# -*- coding: utf-8 -*-
'''
    date:2017-05-28 hqk
    spider newadd rest class
'''

import gearman
import redis
from urlparse import urljoin
import requests
import json

from common.xms_json import MyJson
from dianp_access import DianpAccess, DianpRequest
from config.setting import RESTUPDATE, TASKBLOCK, SPIDERSENSUS_REDIS, RESTPICKEY ,RESTINFOKEY, RESTREVIEWKEY, RESTCLOSE, RESTINFO_REDIS, RESTUPDATEREVIEW, RESTBILLBOADKEY, RESTINFOCHECK_KEYS,RESTCLOSE
from dianp_task import TaskStyle, TaskGearman, TaskProtocol, TaskModel
from dianp_census import DianpCensus
from dianp_parse import DianpParse
from dianp_oss import DianpOss
from dianp_log import DianpLog

import sys 
reload(sys) 
sys.setdefaultencoding("utf-8")


MutiTaskNum = 10

class DianpUpdateTask(TaskModel, TaskGearman):
    def __init__(self):
        super(DianpUpdateTask, self).__init__()
        self.dianpAccess = DianpAccess()
        #self.taskStyleO = taskStyle()
        #self.taskGermanO = TaskGearman()
        self.DianpCensusO = DianpCensus()
        self.dianpLogO = DianpLog('_program.log')
        self.client_init()
        
    
    #
    def __get_req_url(self, start, end):
        restIds = self.dianpAccess.get_restId(start, end)
        #detailUrlList = self.dianpAccess.bulid_restdetail_url(start, end)
        return restIds
        
        
    def __parse_res(self, res):
        allData = []
        state, resJ = myJson.jsonLoads(res)
        if state == False:
            return 
        results = resJ['results']
        for item in results:
            pData = TaskProtocol.parse_update_protocol(item)
            allData.append(pData)
        return allData
        
    #获取新增任务,下发任务
    def send_update_task(self, updateList):
        taskList = []
        updateList = updateList[138:238]
        reqTasks = [updateList[i:i+MutiTaskNum] for i in range(0,len(updateList),MutiTaskNum)]
        for item in reqTasks:
            taskData = TaskStyle.task_style(RESTUPDATE, item)
            taskList.append(taskData)
        print taskList
        res = self.client_multiple_task(taskList)
        return res
        
    
    #解析任务结果, 记录到当天的redis数据
    def parse_task_result(self, res):
        allDatas = TaskProtocol.parse_newadd_protocol(res)
        for allData in allDatas:
            for restid in allData['restids']:
                logStr = "update the restid :[" + restid + "]"
                self.dianpLogO.Log(logStr)
            
    
    def main(self):
        start = 0
        end = TASKBLOCK
        updateList =  self.__get_req_url(start, end)
        while updateList != []:
            print updateList[:10]
            self.dianpLogO.Log('send update tasks :' + str(len(updateList)))
            res = self.send_update_task(updateList)
            self.parse_task_result(res)
            start = end 
            end = start + TASKBLOCK
            updateList =  self.__get_req_url(start, end)
            
        
        
class DianpUpdateHandle(TaskGearman):
    def __init__(self):
        super(DianpUpdateHandle, self).__init__()
        self.worker_init()
        self.updateProxieTimes = 0
        self.dianpRequestO = DianpRequest()
        self.dianpAcessO = DianpAccess()
        self.dianpCensusO = DianpCensus()
        self.dianpOssO = DianpOss()
        #self.dianpParseO = DianpParse()
        
        self.pool = redis.ConnectionPool(host=RESTINFO_REDIS['host'], port=RESTINFO_REDIS['port'], db=RESTINFO_REDIS['db'])
        self.rds = redis.Redis(connection_pool=self.pool)
        
        
    
    def start(self):
        self.worker_register(RESTUPDATE, self.handle)
        self.worker_start()

    def clean_data(self, tagList):
        allPids = []
        for item in tagList:
            if u'全部图片' in  item['tag']:
                allPids = item['pids']
                break
        
        for item in tagList:
            if u'全部图片' in item['tag']:
                continue
            for id in item['pids']:
                if id in allPids:
                    allPids.remove(id)
       
    def download_all_big_pic(self, restid, tagList):
        for item in tagList:
            tag = item['tag']
            picIds = item['pids']
            print tag
            if tag == u'全部图片':
                tag = u'其他'
            for item2 in picIds:
                picid = item2['id']
                url = item2['url']
                if url == '':
                    continue
                if 'https' in url:
                    import re
                    url = url.replace('https', 'http')
                    url = re.findall('(.*jpg)', url)[0]
                res = requests.get(url)
                if res.status_code == '200':
                    continue
                pathName = self.dianpOssO.bulid_oss_path(restid, tag, picid) 
                self.dianpOssO.upload(pathName, res.content)

    def get_all_tag_picid(self, restid, tagList):
        num = 0
        for item in tagList:
            item['pids'] = []
            tagUrl = urljoin(self.dianpAcessO.DianpHost, item['url'])
            state, res = self.dianpRequestO.request_get(tagUrl)
            pageNum = DianpParse.get_max_pagenum(res.content)
            picPageList =  self.dianpAcessO.build_pic_url_tag(tagUrl, pageNum, 1)
            picIds = self.get_small_pic_url(restid, picPageList)
            for id in picIds:
                if not self.picExits(restid, id):
                    item['pids'].append(id)
                    num = num + 1
        allPids = []
        for item in tagList:
            if u'全部图片' in  item['tag']:
                allPids = item['pids']
                break

        for item in tagList:
            if u'全部图片' in item['tag']:
                continue
            for id in item['pids']:
                if id in allPids:
                    allPids.remove(id)
        
        return num
            
        
    def get_small_pic_url(self, restid, picPageList):
        picIds = []
        for picUrl in picPageList:
            print picUrl
            state, res = self.dianpRequestO.request_get(picUrl)
            if state == False:
                 continue
            picId = DianpParse.get_restpic(res.content)
            '''
            for pid in picId:
                if self.picExits(restid, pid['id']):
                    print 'find the spidered pid'
                    return picIds + picId
            '''
            picIds = picIds + picId
        return picIds
    
    def get_big_pic_url(self, tagLists):
        tagPics = []
        for item in tagLists:
            tag = {}
            tag['tag'] = item['tag']
            pids = item['pids']
            tag['pids'] = []
            for item in pids:
                id = item['id']
                picUrl = self.dianpAcessO.build_big_pic_url(id)
                state,res = self.dianpRequestO.request_get(picUrl)
                bigPicUrl = DianpParse.get_big_pic(res.content)
                tag['pids'].append({'id':id, 'url':bigPicUrl, 'intro':item['intro']})
            tagPics.append(tag)
        return tagPics

    def get_all_reviews(self, restid):
        #newReviewid = self.restReview(restid)
        reviewUrl = self.dianpAcessO.build_review_url(restid, 1)
        state, res = self.dianpRequestO.request_get(reviewUrl)
        if state == False or res.status_code != 200:
            self.dianpCensusO.update_proxie_times()
            return []
        pageNum = DianpParse.get_max_pagenum(res.content)
        allUrl = self.dianpAcessO.build_all_review_url(restid, pageNum, 1)
        allReviewList = []
        for url in allUrl:
            state, res = self.dianpRequestO.request_get(url)
            if state == False:
                continue
            reviewList = DianpParse.get_review(res.content)
            reviewListU = []
            for item in reviewList:
                if self.restReview(restid, item['id']):
                    continue
                else:
                    allReviewList.append(item)
        return allReviewList
          
    def handle(self, gearmanWorker, gearmanJob):
        requestData = gearmanJob.data
        requestDataS = json.loads(requestData)
        updateList = []
        for restid in requestDataS:
            detailUrl = self.dianpAcessO.build_rest_detail(restid)
            print detailUrl
            state, res = self.dianpRequestO.request_get(detailUrl)
            if state == False or res.status_code != 200:
                #self.dianpCensusO.update_proxie_times()
                state, res = self.dianpRequestO.request_get(detailUrl)
                continue
            flag = 3
            while DianpParse.check_page_404(res.content) == True and flag > 0:
                state, res = self.dianpRequestO.request_get(detailUrl)
                while state == False:
                    state, res = self.dianpRequestO.request_get(detailUrl)
                #flag = flag - 1
            if DianpParse.is_closed_rest(res.content) == True:
                '''
                restInfo = DianpParse.get_restinfo(res.content)
                #获取区域 
                districtUrl = self.dianpAcessO.build_district_url(restInfo['restinfo']['cityId'], restInfo['restinfo']['mainRegionId'])
                state, res = self.dianpRequestO.request_get(districtUrl)
                if state == False or res.status_code != 200:
                    continue
                restInfo['restinfo']['districtId'] = DianpParse.get_res_districtid(res.content)
                 
                self.dianpCensusO.rest_close(restid, restInfo)
                '''
                continue
            else:
                restInfo = DianpParse.get_restinfo(res.content)
                #获取点评标签
                userThink = DianpParse.get_user_think(res.content)
                restInfo['userThink'] = userThink
                #获取点评
                reviewList = DianpParse.get_review_mainpage(res.content)
                if len(reviewList) > 0:
                    self.dianpCensusO.rest_update_review(restid, reviewList)
                #推荐菜
                recmmendParams = DianpParse.get_recommend_params(restInfo['restinfo'])
                recommenUrl = self.dianpAcessO.build_rest_recomend()
                state, res = self.dianpRequestO.request_get(recommenUrl, params=recmmendParams)
                if state == False or res.status_code != 200:
                    self.dianpCensusO.update_proxie_times()
                    continue

                recommendFood = DianpParse.get_recommend_food(res.content)
                for food in recommendFood:
                    foodUrl = self.dianpAcessO.build_food_tag_url(restid, food['name'])
                    state, res = self.dianpRequestO.request_get(foodUrl)
                    if res.status_code == 404:
                        continue
                    DianpParse.get_food_tag(res.content)
                    food['foodLabel'] = DianpParse.get_food_tag(res.content)
                restInfo['recommendFood'] = recommendFood
                #获取榜单
                billboadParams = DianpParse.get_billboad_params(restInfo['restinfo'])
                billboadUrl = self.dianpAcessO.build_rest_billboad()
                state, res = self.dianpRequestO.request_get(billboadUrl, params=billboadParams)
                if state == False or res.status_code != 200:
                    continue

                billBoad = DianpParse.get_rest_billboad(res.content)

                '''
                newBillBoad = []
                for item in billBoad:
                    if self.billboadExits(restid, item['listId']):
                            newBillBoad.append(item)
                print billBoad,'###'
                '''
                newBillBoad = billBoad
                restInfo['billBoad'] = newBillBoad
                #获取区域 
                districtUrl = self.dianpAcessO.build_district_url(restInfo['restinfo']['cityId'], restInfo['restinfo']['mainRegionId'])
                state, res = self.dianpRequestO.request_get(districtUrl)
                if state == False or res.status_code != 200:
                    continue
                restInfo['restinfo']['districtId'] = DianpParse.get_res_districtid(res.content)
                
                state, censusInfo = self.check_restInfo(restid, restInfo)
                if state == False:
                    self.dianpCensusO.rest_update_info(restid, censusInfo)

                picUrl = self.dianpAcessO.build_rest_pic_url(restid)
                state, res = self.dianpRequestO.request_get(picUrl)
                if state == False or res.status_code != 200:
                    continue
                tagList = DianpParse.get_tag_list(res.content)
                
                if len(tagList) == 0:
                    continue
                
                picnum = self.get_all_tag_picid(restid, tagList)
                print picnum, ' new addd pics'
                if picnum == 0:
                    continue
                tagPics = self.get_big_pic_url(tagList)
                self.download_all_big_pic(restid, tagPics)
                self.dianpCensusO.rest_update_pic(restid, tagPics)

                updateList.append(restid)
        return json.dumps({'restids':updateList})
        
    def check_restInfo(self, restid, newInfo):
        cencusInfo = {'restinfo':None, 'billBoad':None, 'recommendFood':None,'userThink':None, 'diffKeys':None}
        oldInfo = self.rds.get(RESTINFOKEY(restid))
        
        if oldInfo == None:
            state, diffKeys = self.dianpCensusO.check_restInfo(restid, newInfo['restinfo'])
            newInfo['diffKeys'] = diffKeys
            return False, newInfo
        diffKeys = []
        oldInfo = json.loads(oldInfo)
        diffInfo = {}
        for k,v in newInfo['restinfo'].items():
            if k in RESTINFOCHECK_KEYS:
                if oldInfo['restinfo'][k] != v:
                    diffInfo[k] = v
                    diffKeys.append(k)
        if diffInfo != {}:
            cencusInfo['restinfo'] = newInfo['restinfo']
            cencusInfo['diffKeys'] = diffKeys

        if newInfo['billBoad'] != oldInfo['billBoad']:
            cencusInfo['billBoad'] = newInfo['billBoad']
        

        if newInfo['recommendFood'] != oldInfo['recommendFood']:
            cencusInfo['recommendFood'] = newInfo['recommendFood']

        if newInfo['userThink'] != oldInfo['userThink']:
            cencusInfo['userThink'] = newInfo['userThink']
        
        return False, cencusInfo
                
    def restExits(self, restid):
        if self.rds.sismember(REST_LIST_KEY, restid) == 0:
            return False
        else:
            return True

    def restReview(self, restid, reviewid):
        if self.rds.sismember(RESTREVIEWKEY(restid), reviewid) == 0:
            return False
        else:
            return True

    def billboadExits(self, restid, listid):
        if self.rds.sismember(RESTBILLBOADKEY(restid), listid) == 0:
            return False
        else:
            return True
            
    def picExits(self, restid, picid):
        if self.rds.sismember(RESTPICKEY(restid), picid) == 0:
            return False
        else:
            return True
        
#dp = DianpUpdateHandle()
#dp.handle('','')
